<?php
 mb_internal_encoding("utf-8");
 //gc_enabled();
 require "../document/class_corpus.php";
 require "../document/class_document.php";
 require "../document/class_document_xml.php";

 require "../files/tool_files_dirs.php";

 require "../misc/class_tag.php";
 require "../misc/tool_color.php";

 require "../tree/class_tree.php";
 require "../tree/class_tree_xhtml.php";
 require "../tree/class_structure.php";
 require "../tree/tool_tree.php";

 require "../array/tool_array.php";

 require "../pear_graph/tool_pear_graph.php";

 require "../maths/tool_maths.php";

 require "../string/tool_string.php";
 require "../string/class_segment.php";

/*
$array_ids_europa=tool_corpus::europa_get_all_ids("../xml/europa_complete_repair/");

$cpt_multidoc=0;
$multidoc_lg=array();
$array_lg=array();

foreach($array_ids_europa as $id_europa){
  $corpus=tool_corpus::get_corpus_regexp("../xml/europa_complete_repair/",$id_europa);
  $array_doc=$corpus->__get_array_doc();
  $cpt_multidoc++;
  foreach($array_doc as $document){
    $id_doc=mb_strcut($document->__get_id(),-6,2);
    $multidoc_lg[$id_europa][$id_doc]=1;
    $array_lg[$id_doc]++;
//    echo "$id_doc\n";
  } 

}


//$lg_candidates=array("cs","da","de","el","en","es","et","fi","fr","hu","it","lt","lv","mt","nl","pl","pt","sk","sl","sw");
//$lg_candidates=array("cs","da","de","el","en","es","et","fi","fr","hu","it","lt","lv","nl","pl","pt","sk","sl","sw");
$lg_candidates=array("cs","da","de","el","en","es","et","fi","fr","hu","it","lt","lv","nl","pl","pt","sl","sw");

//print_r($multidoc_lg);
//print_r($array_lg);
//die();

$res=array();
foreach($multidoc_lg as $multi=>$array_lg){
  $flag=true;
  foreach($lg_candidates as $lg){
    if(!array_key_exists($lg,$array_lg)){
      $flag=false;
      break;
    }
  }
  if($flag)$res[]=$multi;
}
*/
$lg_candidates=array("cs","da","de","el","en","es","et","fi","fr","hu","it","lt","lv","nl","pl","pt","sl","sw");
$dir="../xml/europa_complete_repair/";

$res=tool_corpus::get_unified_europa($dir,$lg_candidates);


//print_r($res);
$str=implode("\",\n      \"",$res);
$str=rtrim($str,",");
echo "array(\"$str\")";
die();

print_r($multidoc_lg);

die();
/*
foreach(){

}
*/

foreach($array_lg as $lg=>$cpt){
 if($cpt>=40) $res.="\"$lg\",";
} 
 $res=rtrim($res,",");
echo "array(".$res.")";

//print_r($multidoc_lg);

die();

$array_corpus_groupe=array();
$start=microtime(true);
$cpt=0;
$taille=0;
$array_bloc_str=array();
$factory_mot=new factory_mot();
foreach($array_ids_europa as $id_europa){
  $corpus_count=tool_corpus::get_corpus_regexp("../xml/europa/",$id_europa);
  $count=count($corpus_count->__get_array_doc());
  if($count>=20){
  $corpus=tool_corpus::get_corpus_regexp("../xml/europa/",$id_europa."(fr|es|en|fi|de|it|sk|el|mt)");
//  $corpus=tool_corpus::get_corpus_regexp("../xml/europa/",$id_europa."(fr|fi|es|de|en)");
  $array_doc=$corpus->__get_array_doc();
//  if($cpt>20) break;
  foreach($array_doc as $document){
//    echo "\n".$document->__get_id()."\n";
    $id_doc=mb_strcut($document->__get_id(),-6,2);
    $structure=$document->__get_structure();
    $signature=$structure->__get_signature();
//    unset($structure);
//    unset($document);
    tool_tree::cleanstructure(&$signature);
    $signature->__recalc_idn();

    $test=new RecursiveIteratorIterator($signature,RecursiveIteratorIterator::SELF_FIRST);

    foreach($test as $id_bloc=>$bloc_xhtml){
      $array_bloc_str[] = tool_tree::tree_xhtml2texte($bloc_xhtml->__get_value());
      $factory_mot->get_all_mots($array_bloc_str[count($array_bloc_str)-1],$id_doc);
    }
  }
  }
}
//    echo  "\r".number_format(memory_get_usage(),0,".",",");
//$all_seg=$factory_mot->__get_array_segment();
$end=microtime(true)-$start;
//echo "\n total :: $end $cpt\n";

$factory_mot->init_etiquetage();
//$start=microtime(true);
$factory_mot->init_chunk();
//$end=microtime(true)-$start;
//echo "\n total :: $end";
//die();

function clique_str($array_dico){
 $res=array();

 foreach($array_dico as $id_doc=>$array_str){
  foreach($array_str as $str=>$freq){
    $res[$freq][$id_doc][]=$str;
  }
 }
 return $res;
}
/*
$clique=clique_str($factory_mot->__get_dictionnaire_doc_freq());

foreach($clique as $freq=>$array){
  echo $freq."\n";
  if(count($array)==5){
    foreach($array as $id_doc=>$array_str){
      echo "   $id_doc\n      ";
      foreach($array_str as $str){
        echo $str." ";
      }
      echo "\n";
    }
  }
}
*/

function clique_vector_str($array_dico){
 $array_res=array();
 $array_vector=array();

 foreach($array_dico as $id_doc=>$array_str){
  foreach($array_str as $str=>$array_bloc){
    $flag=false;
    foreach($array_vector as $id=>$vector){
      if($array_bloc==$vector){
        $array_res[$id][$id_doc][]=$str;
        $flag=true;
        break;
      }
    }
    if(!$flag){
//      print_r($array_bloc);
      $new_id_vector=array_push($array_vector,$array_bloc)-1;
      $array_res[$new_id_vector][$id_doc]=array($str);
    } 
  }
 }
 return array($array_res,$array_vector);
  
}

//$doc_bloc_freq=$factory_mot->__get_dictionnaire_doc_bloc_freq();
//print_r($doc_bloc_freq);
//die();

//$repartition_relative=$factory_mot->__get_relative_repartition_bloc();
$repartition_relative=$factory_mot->__get_relative_doc_bloc_freq();

//print_r($repartition_relative);

//die();

list($array_res,$array_vector)=clique_vector_str($repartition_relative);
//die();
//print_r($array_res);
//die();
//
foreach($array_vector as $id_vector=>$vector){
  if(count($array_res[$id_vector])==9){
    echo "=======\n";
    foreach($array_res[$id_vector] as $id_doc=>$array_str){
      echo "   $id_doc\n      ";
      foreach($array_str as $str){
        echo "$str ";
      }
      echo "\n";
    }
  }

}
die();

foreach($clique as $freq=>$array){
  echo $freq."\n";
  if(count($array)==9){
    foreach($array as $id_doc=>$array_str){
      echo "   $id_doc\n      ";
      foreach($array_str as $str){
        echo $str." ";
      }
      echo "\n";
    }
  }
}

die();
$tool_graph=new Tool_pear_graph();

//$repartition_chunk=$factory_mot->__get_repartiton_chunk();
$repartition_segment=$factory_mot->__get_repartition_segment();
//print_r($repartition_segment);
//die();
$res_v=$res_all=$res_p=$res_i=$zipf=array();
foreach($repartition_segment as $id_doc=>$array_bloc){
  foreach($array_bloc as $id_bloc=>$array_id_segment){
    foreach($array_id_segment as $id_segment){
      $mot=$factory_mot->__get_word($id_segment);
      $len=$mot->__get_len();
      $type=$mot->__get_attribute("type");
      if($type=="v") $res_v[$len]++;
      if($type=="p") $res_p[$len]++;
      if($type=="i") $res_i[$len]++;
      $id_doc=mb_strcut($id_doc,-6,2);
      $res_all[$id_doc][$len]++;
      $zipf[$id_doc][$mot->__tostring()]++;
    }
  }
}
/*
$len_chunk=array();
$repartition_chunk=$factory_mot->__get_repartition_chunk();
//print_r($repartition_chunk);
//die();
foreach($repartition_chunk as $id_doc=>$array_bloc){
  $id_doc=mb_strcut($id_doc,-6,2);
  foreach($array_bloc as $id_bloc=>$array_chunk){
    foreach($array_chunk as $array_id_segment){
      $len=0; 
      foreach($array_id_segment as $id_segment){
        $mot=$factory_mot->__get_word($id_segment);
        //if($mot->__get_attribute("type")!="v")
          $len+=$mot->__get_len();
      }
      $len_chunk[$id_doc][$len]++;
    }
  }
}
*/
//die();


foreach($zipf as $id_doc => $res_zipf){
  $res_zipf=tool_maths::own_zipf($res_zipf);

  $tool_graph->__add_array_x(array_keys($res_zipf[0]));
  $tool_graph->__add_array_y(array_values($res_zipf[0]));
  $tool_graph->__add_plot_title("zipf log*log : $id_doc");

/*
  $tool_graph->__add_array_x(array_keys($res_zipf[2]));
  $tool_graph->__add_array_y(array_values($res_zipf[2]));
  $tool_graph->__add_plot_title("zipf lg*freq : $id_doc");
*/
/*
  $tool_graph->__add_array_x(array_keys($res_zipf[2]));
  $tool_graph->__add_array_y(array_values($res_zipf[2]));
  $tool_graph->__add_plot_title("zipf lg*freq : $id_doc");
*/ 
/*
  $res_zipf=tool_maths::zipf($res_zipf);
  $tool_graph->__add_array_x(array_keys($res_zipf));
  $tool_graph->__add_array_y(array_values($res_zipf));
  $tool_graph->__add_plot_title("zipf log*log : $id_doc");
*/
}

/*
foreach($res_all as $id_doc => $res){
  $res=tool_maths::zipf($res);
  $tool_graph->__add_array_x(array_keys($res));
  $tool_graph->__add_array_y(array_values($res));
  $tool_graph->__add_plot_title("zipf log*log bis : $id_doc");
}
*/
/*
foreach($res_all as $id_doc => $len_word){
//  if($id_doc=="fi"){
    ksort($len_word);
    $tool_graph->__add_array_x(array_keys($len_word));
    $tool_graph->__add_array_y(array_values($len_word));
    $tool_graph->__add_plot_title("len_word : $id_doc");
//  }
}

foreach($len_chunk as $id_doc => $len){
//  if($id_doc=="fr"){
    ksort($len);
    $tool_graph->__add_array_x(array_keys($len));
    $tool_graph->__add_array_y(array_values($len));
    $tool_graph->__add_plot_title("len_chunk : $id_doc");
//  }
}
*/
//die();
//print_r($zipf_log);
//die();
//print_r($zipf);
//die();
//$tool_graph->__add_array_x(array_keys($res_all));
//$tool_graph->__add_array_y(array_values($res_all));
//$tool_graph->__add_array_x(array_keys($zipf));
//$tool_graph->__add_array_y(array_values($zipf));
//$tool_graph->__add_array_x(array_keys($zipf_log_fi));
//$tool_graph->__add_array_y(array_values($zipf_log_fi));
//$tool_graph->__add_plot_title("zipf finnois log*log");
//$tool_graph->__add_plot_title("zipf");


/*
$tool_graph->__add_array_x(array_keys($res_v));
$tool_graph->__add_array_x(array_keys($res_i));
$tool_graph->__add_array_x(array_keys($res_p));
*/
/*
$tool_graph->__add_array_y(array_values($res_v));
$tool_graph->__add_array_y(array_values($res_i));
$tool_graph->__add_array_y(array_values($res_p));
*/
//$tool_graph->__add_title("droite de régression sur le rapport longueur*effectif de la loi de zipf des documents");
//$tool_graph->__add_title("rapport longueur*effectif de la loi de zipf des documents");
$tool_graph->__add_title("loi de zipf");
//$graph=$tool_graph->plot03();
$graph=$tool_graph->plot_log_log01();
//$graph=$tool_graph->plot_line_regression();
//$graph=$tool_graph->plot_dot_regression();
//$graph=$tool_graph->plot03();
ob_start();
$graph->done();
$graph_svg=ob_get_contents();
Tool_files::file_write("./out.svg",$graph_svg);
ob_clean();
?>
